/*******************************************************************************
    文件名称： 
	开发者：孙磊
	开发时间地点：2013.12.24 
	邮箱：zhongyuansunlei@163.com
	版本信息：
		
********************************************************************************/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <pthread.h>
#include <fcntl.h>
#include <signal.h>
#include <netdb.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include "common.h"
#include "strutil.h"
#include "nodearray.h"
#include "wrapepoll.h"
#include "queue.h"
#include "hash.h"
#include "http.h"
#include "spider.h"
#include "html.h"
#include "log.h"


#define MAX_EVENTS	FOUR_K_SIZE

cmd g_ocmd ;
spidercfg g_ospidercfg;
runtimeinfo g_oruntimeinfo;
httpinfo g_ohttpinfo;

void* purl_crawler(void* arg);
void* purl_dispatcher(void* arg);


void print_url(void* pe)
{
	urlinfo* p = (urlinfo*)pe;
	
	Trace("%s\n", p->m_surl);
}

int url_cpy(queue_elem* p, void* q)
{
	char* x;
	urlinfo *y;
	
	x = (char*)q;
	y = (urlinfo*)malloc(sizeof(urlinfo));
	if(y == NULL)
		return -1;
	memset(y->m_surl,0, HALF_K_SIZE);
	strcpy(y->m_surl, x);
	p->m_pval = y;
	
	return 0;
	
}

void url_free(void* p)
{
	queue_elem *y = (queue_elem*)p;
	free(y->m_pval);
	y->m_pval=NULL;
	
	return ;
}

void url_print(void* p)
{
	char* q = (char*)p;
	Trace("q = %s\n", q);
}


int iread_pipe(int ifd, char* pbuf, int isize)
{
	char *p = pbuf;
	int ilen, ileft, itotal; 
  ileft = HALF_UNIT_SIZE;
	ilen = 0;
	
	//先读长度，再度url
	memset(pbuf,0,HALF_UNIT_SIZE+1);
	while(ileft > 0){
		ilen = read(ifd, p+ilen, ileft);
		if(ilen >= 0) {
			ileft-=ilen;
		} else {
			Trace("ifd = %d, errno = %d\n", ifd, errno);
			return -1;
		}
	}
	
	ilen = 0;
	itotal = ileft = atol(pbuf);
	p = pbuf;
	memset(pbuf,0,ileft+1);
	while(ileft > 0){
		ilen = read(ifd, p+ilen, ileft);
		if(ilen >= 0) {
			ileft-=ilen;
		} else {
			Trace("errno = %d\n", errno);
			return -1;
		}
	}
	
	return itotal;
}

//写到缓冲区的内容包括url的8位长度，紧接着是url
int iwrite_pipe(int ifd, char* pvalue)
{
	char ssend[HALF_K_SIZE], *p;
	int ilen, ileft;
	memset(ssend, 0, HALF_K_SIZE);
	ilen =  strlen(pvalue);
	snprintf(ssend, HALF_K_SIZE, "%08d%s", ilen, pvalue);
	
	p = ssend;
	ilen = 0;
	ileft = strlen(ssend);
	while (ileft > 0) {
		ilen = write(ifd, p + ilen, ileft);
		if(ilen >= 0) {
			ileft-=ilen;
		} else {
			Trace("errno = %d\n", errno);
			return -1;
		}
	}
	
	return strlen(ssend);
}

int iparsecfg(spidercfg* pspidercfg)
{
	FILE* pfile = NULL;
	char sfile[HALF_K_SIZE], sbuffer[K_SIZE];
	int ilen,irtn;
	ilen = irtn = 0;
	char *p, *q;
	memset(sfile, 0, HALF_K_SIZE);
	memset(sbuffer, 0, K_SIZE);
	char* pcfg = getenv("SUN_SPIDER_INI");
	if(pcfg == NULL) {
		Trace("未配置环境变量SUN_SPIDER_INI，当前读取本目录下的配置文件sun.ini\n");
		strcpy(sfile, "sun.ini");
	} else {
		Trace("使用环境变量SUN_SPIDER_INI的配置文件，配置文件为%s\n", pcfg);
		strcpy(sfile, pcfg);
	}
	
	pfile = fopen(sfile, "r");
	if(pfile == NULL) {
		Trace("打开文件%s失败，错误码errno=%d\n", sfile, errno);
		return RC_FAILURE;
	}
	
	while(fgets(sbuffer, K_SIZE, pfile)) {
		if(sbuffer[0] == '#')// 注释行
			continue;
		
		sstr_trim(sbuffer);
		ilen = strlen(sbuffer);
		if(ilen<=0)
			continue;
		p=sbuffer;
		q=strstr(sbuffer, "=");
		if(q==NULL) {
			Trace("解析配置文件失败!\n");
			irtn = RC_FAILURE;
			break;
		}
		
		if(strncmp(p, "WORKER_NUM", strlen("WORKER_NUM")) == 0) {
			if(*(q+1) == '\0') {
				Trace("未配置WORKER_NUM项!\n");
				irtn = RC_FAILURE;
				break;
			}
			
			pspidercfg->m_imaxworker = atoi(q+1);
		} else if(strncmp(p, "INIT_URL", strlen("INIT_URL")) == 0) {
			if(*(q+1) == '\0') {
				Trace("未配置INIT_URL项!\n");
				irtn = RC_FAILURE;
				break;
			}
			
			strcpy(pspidercfg->m_siniturl, q+1);
		} else if (strncmp(p, "USER_AGENT", strlen("USER_AGENT")) == 0) {
			if(*(q+1) == '\0') {
				Trace("未配置INIT_URL项!\n");
				irtn = RC_FAILURE;
				break;
			}
			
			strcpy(pspidercfg->m_suseragent, q+1);
		} else if (strncmp(p, "FILE_ARCHIVE_DIR", strlen("FILE_ARCHIVE_DIR")) == 0) {
			if(*(q+1) == '\0') {
				Trace("未配置INIT_URL项!\n");
				irtn = RC_FAILURE;
				break;
			}
			
			strcpy(pspidercfg->m_sfilearchivedir, q+1);
		} else {
			Trace("解析配置文件失败!\n");
			irtn = RC_FAILURE;
			break;
		}
	}
	
	fclose(pfile);
	
	return irtn;
}


void signal_register()
{
	signal(SIGINT, SIG_IGN);
}

int iinitspider()
{
	int irtn, i;
	irtn = i = 0;
	
	if(iparsecfg(&g_ospidercfg)) {
		Trace("iparsecfg 失败\n");
		return RC_FAILURE;
	}
	
	if(g_ospidercfg.m_imaxworker > MAX_THREADS_NUM) {
		g_ospidercfg.m_imaxworker = MAX_THREADS_NUM;
	}

	
	g_oruntimeinfo.m_imainpfd = (int*) malloc(sizeof(int)*g_ospidercfg.m_imaxworker*2);	
	g_oruntimeinfo.m_ppfd = (int*) malloc(sizeof(int)*g_ospidercfg.m_imaxworker*2);	
	g_oruntimeinfo.m_pepfd = (int*) malloc(sizeof(int)*g_ospidercfg.m_imaxworker);	
	g_oruntimeinfo.m_ptid = (pthread_t*) malloc(sizeof(pthread_t)*g_ospidercfg.m_imaxworker);
	if((g_oruntimeinfo.m_imainpfd == NULL)||(g_oruntimeinfo.m_ppfd == NULL)||(g_oruntimeinfo.m_ptid == NULL)||(g_oruntimeinfo.m_pepfd == NULL)) {
		Trace("申请内存失败！\n");
		return RC_NOSPACE;
	}
	
	if(iinit_nodearray(&g_oruntimeinfo.m_purlcontainer, TWO_K_SIZE)) {
		return RC_FAILURE;
	}
	
	g_oruntimeinfo.m_pressize = (int*)malloc(g_ospidercfg.m_imaxworker*sizeof(int));
	if(g_oruntimeinfo.m_pressize == NULL) {
		return RC_NOSPACE;
	}

	// initial	
	for(i=0;i<g_ospidercfg.m_imaxworker;i++) {
		if(pipe(g_oruntimeinfo.m_imainpfd+i*2) == -1) {
			Trace("创建管道失败，错误码=%d\n", errno);
			return RC_FAILURE;
		}
		if(pipe(g_oruntimeinfo.m_ppfd+i*2) == -1) {
			Trace("创建管道失败，错误码=%d\n", errno);
			return RC_FAILURE;
		}
				
		if(ievent_create(&g_oruntimeinfo.m_pepfd[i])) {
			return RC_FAILURE;
		}
		
		if(iinit_nodearray(&g_oruntimeinfo.m_pnarr[i], TWO_K_SIZE)) {
			return RC_FAILURE;
		}
		
		//if(iinit_queue(&g_oruntimeinfo.m_pq[i], TWO_K_SIZE)){
		if(iinit_queue(&g_oruntimeinfo.m_pq[i], EIGHT_K_SIZE)){
			return RC_NOSPACE;
		}
		
		g_oruntimeinfo.m_preq[i] = (char*) malloc(sizeof(char)*K_SIZE);
		if(g_oruntimeinfo.m_preq[i] == NULL) {
			return RC_NOSPACE;
		}
		
		g_oruntimeinfo.m_pres[i] = (char*) malloc(sizeof(char)*M_SIZE);
		if(g_oruntimeinfo.m_preq[i] == NULL) {
			return RC_NOSPACE;
		}
		
		*(g_oruntimeinfo.m_pressize+i) = M_SIZE;
	}
	
	
	signal_register();
	
	return RC_SUCCESS;
}

int ireleasspider()
{
	// 这里要释放所有的内存
	for(int i=0;i<g_ospidercfg.m_imaxworker;i++) {

		if(irelease_nodearray(g_oruntimeinfo.m_pnarr[i], vev_free)) {
			return RC_FAILURE;
		}
		
		if(irelease_queue(g_oruntimeinfo.m_pq[i], url_free)){
			return RC_NOSPACE;
		}
		
		free(g_oruntimeinfo.m_preq[i]);
		free(g_oruntimeinfo.m_pres[i]);
	}
	
	free(g_oruntimeinfo.m_imainpfd);
	free(g_oruntimeinfo.m_ppfd);
	free(g_oruntimeinfo.m_pepfd);
	free(g_oruntimeinfo.m_ptid);
	
return RC_SUCCESS;
}



char* dn2ip(char* pdnsname)
{
	struct hostent *he;
	struct in_addr **addr_list;
	
	if ((he = gethostbyname(pdnsname)) == NULL) {  // get the host info
		herror("gethostbyname\n");
		return NULL;
	}
	
	//返回第一个地址
	addr_list = (struct in_addr **)he->h_addr_list;
	if(addr_list[0])
		return inet_ntoa(*addr_list[0]);
	else 
		return NULL;
}

int icheck_dnsname(httpinfo*phttpinfo, char* pdnsname)
{
	char* p = NULL;
	sstr_trim(pdnsname);
	
	
	if((p = dn2ip(pdnsname)) == NULL) {
		Trace("dn2ip error\n");
		return -1;
	}
	
	strcpy(phttpinfo->m_sdns, pdnsname);
	strcpy(phttpinfo->m_sip, p); 
	phttpinfo->m_iport = 80;  //将来有空改善
	
	return 0;
}

int irun_cmd_mgr()
{
	vurl_receiver();
	
	return RC_SUCCESS;
}


int iparser_cmd(char* pcmd)
{
	//sstr_trim(pcmd);
	char* pcur, *plast, *pbegin;
	pcur = plast = pbegin = pcmd;
	int i = 0;
	
	while(*pcur != '\0') {
		if(*pcur == ' ') {
			if(*plast == ' ') {
				plast = pcur;
				pcur++;
				continue;
			} else { //当前值为空，上一个值为非空，表示一个参数的结束
				if(i == 0) {
					strncpy(g_ocmd.m_scmdname, pbegin, pcur-pbegin);
					g_ocmd.m_scmdname[pcur-pbegin] = '\0';
				} else {
					strncpy(g_ocmd.m_scmdarg[g_ocmd.m_inum], pbegin, pcur-pbegin);
					g_ocmd.m_scmdarg[g_ocmd.m_inum][pcur-pbegin] = '\0';
					g_ocmd.m_inum++;
				}
				i++;
			}
		} else { 
			if(*plast == ' ') {// 当前值非为空，上一个值为空，
				pbegin = pcur;
			} 
		}
		plast = pcur;
		pcur++;
	}
	
	if(pbegin == pcmd) {
		strncpy(g_ocmd.m_scmdname, pbegin, pcur-pbegin);
		g_ocmd.m_scmdname[pcur-pbegin] = '\0';
	}
	
	if((pbegin != pcmd) && (*plast != ' ')) { // 一个参数
		strncpy(g_ocmd.m_scmdarg[g_ocmd.m_inum], pbegin, pcur-pbegin);
		g_ocmd.m_scmdarg[g_ocmd.m_inum][pcur-pbegin] = '\0';
		g_ocmd.m_inum++;
	}
	
	
	return 0;
}

int irun_url_worker()
{
	static pthread_attr_t tattr[MAX_THREADS_NUM];
	int irtn,i, istacksize;
	static int j[MAX_THREADS_NUM];
	i = irtn = 0;
	istacksize = M_SIZE;

	for(i=0;i<g_ospidercfg.m_imaxworker;i++) {
		j[i] = i;	
		if(pthread_attr_init(&tattr[i])) {
			Trace("pthread_attr_init error, errno = %d\n", errno);
			return RC_FAILURE;
		}	
		
		if(pthread_attr_setstacksize(&tattr[i], istacksize)) {
			Trace("pthread_attr_setstacksize error, errno = %d\n", errno);
			return RC_FAILURE;
		}
	
		if(pthread_create(&g_oruntimeinfo.m_ptid[i], &tattr[i], purl_crawler, &j[i])) {
			Trace("pthread_create 出错， errno = %d\n", errno);
			return RC_FAILURE;
		}
	
		pthread_detach(g_oruntimeinfo.m_ptid[i]);
	}
	
	return RC_SUCCESS;
}

int irun_url_mgr()
{
	static pthread_attr_t tattr;
	pthread_t tid;
	int irtn = 0;
	int istacksize = M_SIZE;
	
	if(pthread_attr_init(&tattr)) {
		Trace("pthread_attr_init error, errno = %d\n", errno);
		return RC_FAILURE;
	}
	
	if(pthread_attr_setstacksize(&tattr, istacksize)) {
		Trace("pthread_attr_setstacksize error, errno = %d\n", errno);
		return RC_FAILURE;
	}
	
	if(irtn = pthread_create(&tid, NULL, purl_dispatcher, NULL)) {
		Trace("pthread_create 出错，irtn = %d, errno = %d\n", irtn, errno);
		return RC_FAILURE;
	}
	
	pthread_detach(tid);
	
	return RC_SUCCESS;
	
}

void vurl_receiver()
{//后期改造为定期清理与命令行管理
	char scmd[HALF_K_SIZE];
	while(true) {
		memset(scmd, 0, HALF_K_SIZE);
		memset(&g_ocmd, 0, sizeof(g_ocmd));
		printf("input dns address縠.g \nput 163.com\n");
		printf(">");
		gets(scmd);
		if(iparser_cmd(scmd)) {
			printf("bad entry!\n");
			continue;
		}
		
		if(strcmp(g_ocmd.m_scmdname, CMD_PUT) == 0) {
			char surl[UNIT_SIZE];
			memset(surl, 0, UNIT_SIZE);
			if(g_ocmd.m_inum == 0) {
				printf("命令解析失败!\n");
				continue;
			}
			
			if(strlen(g_ohttpinfo.m_sip) == 0) {
				if(icheck_dnsname(&g_ohttpinfo, g_ocmd.m_scmdarg[0])) {
					printf("%s 转换ip失败!\n", g_ocmd.m_scmdarg[0]);
					continue;
				}
			} else {
				printf("已经存在目标ip\n");
				continue;
			}
			
			strcpy(surl, "/");
			
			if(iwrite_pipe(g_oruntimeinfo.m_imainpfd[1], surl) < 0) {
				Trace("iwrite_pipe error, errno = %d", errno);
				continue;
			}
			
			break;
		} else {// 以后添加其他的解析命令
		
		}
		
	}
	
	while(true) {//后续添加定期维护等工作
		sleep(1);
	}
}


int str_mem_cpy(node* p, void* q)
{
	char* m, *n;
	m = (char*) p->m_pvalue;
	n = (char*) q;
	
	m = (char*) malloc(strlen(n)+1);
	if(m == NULL)
		return -1;
		
	strcpy(m, n);
	*(m+strlen(n)) = '\0';
	p->m_pvalue = m;
	
	return 0;
}

int str_cmp(node* p, void* q)
{
	char* m, *n;
	m = (char*) p->m_pvalue;
	n = (char*) q;
	
	return strcmp(m, n);
}


int str_hash(void* p)
{
	return BKDRHash((char*)p);
}
	
//该url即为在工作线程过滤过后的url，确保已经是本站资源，本函数仅过滤掉已经爬过的url
//返回0 则为过滤掉，返回1 则为不过滤，返回-1 为错误
int ifilter(char* purl) //
{
	static int i = 0;
	i++;
	Trace("ifilter i = %d\n", i);

	return iaddnode_2_nodearray
	(
		g_oruntimeinfo.m_purlcontainer,
		(void*)purl, 
		str_mem_cpy, 
		str_cmp,
		str_hash
	);
}


void* purl_dispatcher(void* arg)
{
	char surl[HALF_K_SIZE];
	int ilastcur = 0;
	int irtn;
	int i,imaxfd;
	fd_set	rset;
	
	while(true) {
		memset(surl, 0, HALF_K_SIZE);
		FD_ZERO(&rset);
        	for(i=0;i<g_ospidercfg.m_imaxworker;i++)
        	{
                	FD_SET(g_oruntimeinfo.m_imainpfd[i*2], &rset);
        	}

        	imaxfd = g_oruntimeinfo.m_imainpfd[(i-1)*2];		

		select(imaxfd + 1, &rset, 0, 0, 0);
		for(i=0;i<g_ospidercfg.m_imaxworker;i++) {
			if(!FD_ISSET(g_oruntimeinfo.m_imainpfd[i*2], &rset))
				continue;
			
			if(iread_pipe(g_oruntimeinfo.m_imainpfd[i*2], surl, HALF_K_SIZE) < 0){
				Trace("read 错误，错误码errno = %d", errno);
				continue;
			}
			
			irtn = ifilter(surl);
			Trace("url = %s, irtn = %d\n", surl, irtn);
			if((irtn == -1) || (irtn == 0)) {		// 该filter只检查该url是否为所爬站的地址，如不是，则不进行爬行
				Trace("非本站url，进行舍弃，url = %s\n", surl);
				continue;
			}
			
			if((ilastcur+1) == g_ospidercfg.m_imaxworker) {
				ilastcur = 0;
			} else {
				ilastcur++;
			}
			
			Trace("dispatcher url = %s, icur = %d, ifd = %d \n", surl, ilastcur, g_oruntimeinfo.m_ppfd[ilastcur*2+1]);
			//均匀分配url
			if(iwrite_pipe(g_oruntimeinfo.m_ppfd[ilastcur*2+1], surl) < 0) {
				Trace("write 错误，错误码errno = %d\n", errno);
				continue;
			}
		}
	}
	
	return NULL;
}


void vurl_process(int ifd, int ievent, void* arg)
{
	reqinfo *preqinfo = (reqinfo*) arg;
	int irtn;
	event oevent, *pevent;
	node * pnode;
	reqinfo *preqinfo2 = NULL;
	
	if(ievent == EPOLLIN) {
		if(ifd == g_oruntimeinfo.m_ppfd[preqinfo->m_icur*2]) { // 说明是管道过来的信息
			int idestifd;	
			pnode = NULL;
			
			//正常情况下url不可能超过1k大小，这里只做1k的清空内存
			memset(g_oruntimeinfo.m_preq[preqinfo->m_icur], 0, K_SIZE);
		
			if(iread_pipe(g_oruntimeinfo.m_ppfd[preqinfo->m_icur*2], g_oruntimeinfo.m_preq[preqinfo->m_icur], K_SIZE) == -1){
					Trace("read 错误，错误码errno = %d\n", errno);
					return;
			}
			//Trace("init read url = %s\n", g_oruntimeinfo.m_preq[preqinfo->m_icur]);
			
			if((idestifd = ihttp_connect(g_ohttpinfo.m_sip, g_ohttpinfo.m_iport)) == -1) {
				Trace("ihttp_connect调用错误，连接%s 端口%i失败!\n", g_ohttpinfo.m_sip, g_ohttpinfo.m_iport);
				return;
			}
			
			fcntl(idestifd, F_SETFL, O_NONBLOCK);
			if(ihttp_send(idestifd, g_oruntimeinfo.m_preq[preqinfo->m_icur], &g_ohttpinfo) == -1) {
				Trace("ihttp_send调用错误，访问[%s]请求失败!\n", g_oruntimeinfo.m_preq[preqinfo->m_icur]);
				return;
			}
			
			preqinfo2 = (reqinfo*)malloc(sizeof(reqinfo));
			if(preqinfo2 == NULL) {
				exit(-1);
			}
			
			memset(&oevent, 0, sizeof(oevent));
			oevent.m_ifd = idestifd;
			if((pnode = paddnode_2_nodearray(g_oruntimeinfo.m_pnarr[preqinfo->m_icur],  (void*)&oevent, iev_mem_alloc_cpy, iev_cmp, iev_hash)) == NULL) {
				exit(-1);
			}
			preqinfo2->m_icur = preqinfo->m_icur;
			strcpy(preqinfo2->m_surl, g_oruntimeinfo.m_preq[preqinfo->m_icur]);
			pevent = (event*)pnode->m_pvalue;
			vevent_set(pevent, idestifd, vurl_process, (void*)preqinfo2, EV_READ);
			if(ievent_add(g_oruntimeinfo.m_pepfd[preqinfo->m_icur], pevent)) {
				Trace("what happened\n");
				exit(-1);
			}
		} else { // 说明是socket 过来的信息
			resinfo oresinfo;
			oevent.m_ifd = ifd;
			long llength = 0;
			
			if((llength = ihttp_rcv(ifd, &g_oruntimeinfo.m_pres[preqinfo->m_icur], 
				&g_oruntimeinfo.m_pressize[preqinfo->m_icur], &oresinfo)) < 0) {
				Trace("访问%s 返回信息出错！\n", preqinfo->m_surl);
			}
			
			
			if(llength > 0) {// 这里解析html 生成url容器 若果url容器大于零，须进行添加可写事件
				Trace("request url =%s, length = %d\n", preqinfo->m_surl, llength);
				if(ifile_archive(g_ospidercfg.m_sfilearchivedir , preqinfo->m_surl,
						 g_oruntimeinfo.m_pres[preqinfo->m_icur], llength)) {
					Trace("ifile_archive出错！\n");
					return;
				}
				Trace("req url = %s. preqinfo->m_icur = %d\n", preqinfo->m_surl, preqinfo->m_icur);	
				iparser_html(g_oruntimeinfo.m_pres[preqinfo->m_icur], 
				g_oruntimeinfo.m_pq[preqinfo->m_icur],
				url_cpy);
				
				//vprint_queue(g_oruntimeinfo.m_pq[preqinfo->m_icur], print_url);
				if(iget_queue_num(g_oruntimeinfo.m_pq[preqinfo->m_icur])>0) {//这里如果url有添加就要设置可写事件；
					oevent.m_ifd = g_oruntimeinfo.m_imainpfd[preqinfo->m_icur*2+1];
					if(!iexist_in_nodearray(g_oruntimeinfo.m_pnarr[preqinfo->m_icur],  (void*)&oevent, iev_cmp, iev_hash)) {
						if((pnode = paddnode_2_nodearray(g_oruntimeinfo.m_pnarr[preqinfo->m_icur],  (void*)&oevent, iev_mem_alloc_cpy, iev_cmp, iev_hash)) == NULL) {
							exit(-1);
						}
						pevent = (event*)pnode->m_pvalue;
						preqinfo2 = (reqinfo*)malloc(sizeof(reqinfo));
						if(preqinfo2 == NULL) {
							exit(-1);
						}
						preqinfo2->m_icur = preqinfo->m_icur;
						Trace("m_icur = %d\n", preqinfo2->m_icur);
						vevent_set(pevent, oevent.m_ifd, vurl_process, (void*)preqinfo2, EV_WRITE);
						if(ievent_add(g_oruntimeinfo.m_pepfd[preqinfo->m_icur], pevent)) {
							Trace("nothing \n");
							exit(-1);
						}
					}
					
				}
			}
		
			oevent.m_ifd = ifd;	
			if(idelnode_from_nodearray(g_oruntimeinfo.m_pnarr[preqinfo->m_icur], (void*)(&oevent),
				vev_free, iev_cmp, iev_hash)) {
				Trace("idelnode_from_nodearray出错！\n");
				return;
			}
			
			ihttp_close(ifd);
		}
	} else if(ievent == EPOLLOUT) {
		int icur = preqinfo->m_icur;
		Trace("icur == %d\n", icur);
		queue_elem* p = pde_queue(g_oruntimeinfo.m_pq[icur]);
		if(p == NULL) { // 没有可写的url了，把事件也清理了
			vreset_queue(g_oruntimeinfo.m_pq[icur]);
			oevent.m_ifd = ifd;
			if(idelnode_from_nodearray(g_oruntimeinfo.m_pnarr[icur], (void*)(&oevent),
				vev_free, iev_cmp, iev_hash)) {
				Trace("idelnode_from_nodearray出错！\n");
				return;
			}
			return;
		}
		
		urlinfo* qurl = (urlinfo*)(p->m_pval);
		Trace("init send url = %s\n", qurl->m_surl);
		if(iwrite_pipe(g_oruntimeinfo.m_imainpfd[icur*2+1], qurl->m_surl) == -1) {
			Trace("write 错误，错误码errno = %d, url = %s\n", errno,  qurl->m_surl);
			return;
		}
	} 

	return;
}

void* purl_crawler(void* arg)
{
	int icur = *((int*)arg);
	event oevent, *pevent;	
	node *pnode = NULL;
	reqinfo *preqinfo = NULL;

	preqinfo = (reqinfo*)malloc(sizeof(reqinfo));
	if(preqinfo == NULL) {
		exit(-1);
	}
	
	memset(&oevent, 0, sizeof(oevent));
	oevent.m_ifd = g_oruntimeinfo.m_ppfd[icur*2];
	preqinfo->m_icur = icur;
	if((pnode = paddnode_2_nodearray(g_oruntimeinfo.m_pnarr[icur],  (void*)&oevent, iev_mem_alloc_cpy, iev_cmp, iev_hash)) == NULL) {
		exit(-1);
	}
	
	pevent = (event*)pnode->m_pvalue;
	vevent_set(pevent, g_oruntimeinfo.m_ppfd[icur*2], vurl_process, (void*)preqinfo, EV_READ);
	if(ievent_add(g_oruntimeinfo.m_pepfd[icur], pevent)) {
		Trace("oh god\n");
		exit(-1);
	}
		
	ievent_loop(g_oruntimeinfo.m_pepfd[icur], 50);
	
	return NULL;
}
